# -*- coding: utf-8 -*-
import re
import scrapy
import time
from scrapy.http import Request
from bs4 import BeautifulSoup as bs
import xml.etree.ElementTree as ET
from dingdian.items import NvdItem, PacketStormSecurityItem, MitreItem
from dingdian.init_mysql import session
import sys
import pymysql

from dingdian.models import VulNvd, VulPacketStormSecurity
from dingdian.schedule import schedule

reload(sys)
sys.setdefaultencoding( "utf-8" )

class Myspider(scrapy.Spider):

    name = 'mitre'
    allowed_domains = ['cve.mitre.org']
    bash_url = 'http://cve.mitre.org/data/downloads/index.html'
    cvePattern = re.compile(r'CVE-\d{4}.\d+')
    timePattern = re.compile(r'\d{4}-\d{2}-\d{2}')
    pagePattern = re.compile(r'\b\d+\D?\d+')
    cvss = '{http://www.icasi.org/CVRF/schema/vuln/1.1}'
    headers = {
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
        'Accept-Language': 'zh-CN,zh;q=0.9',
        'Cache-Control': 'max-age=0',
        'Pragma': 'no-cache',
        'Referer': 'http://cve.mitre.org',
        'Host' : 'cve.mitre.org',
        'Upgrade-Insecure-Requests': '1',
        'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36'
    }
    pageNum = 0
    vulNum = 0
    totalVuls = 0
    currentVuls = 0.0

    def start_requests(self):
        url = self.bash_url
        yield Request(url,callback= self.parse,headers=self.headers)

    def parse(self, response):
        #print(response.text)
        pages = bs(response.text,'lxml').find('td',attrs={'class':'borderbottomcellboarder','style':'width: 40%'}).find_all('a')
        self.pageNum = len(pages)

        self.totalVuls = int(bs(response.text,'lxml').find('div',id='HeaderBar').find('a').get_text().strip())
        for i in range(self.pageNum):
            url = 'http://cve.mitre.org'+pages[i]['href']
            yield Request(url,callback=self.get_vul)


    def get_vul(self,response):
        #print response.text

        root = ET.fromstring(response.text)

        vuls = root.findall(self.cvss+'Vulnerability')
        #print vuls
        self.vulNum = len(vuls)
        for i in range(self.vulNum):

            item = MitreItem()
            # item = PacketStormSecurityItem()
            item['url'] = response.url
            item['cveID'] = vuls[i].find(self.cvss+'CVE').text
            item['title'] = item['cveID']
            notes = vuls[i].find(self.cvss + 'Notes').findall(self.cvss + 'Note')
            item['message'] = notes[0].text
            try:
                item['publishTime'] = notes[1].text
            except:
                item['publishTime'] = None
            if item['publishTime'] == None:
                yield None
            else:
                self.currentVuls += 1.0
                schedule(self.totalVuls, self.currentVuls)
                try:
                    item['updateTime'] = notes[2].text
                except:
                    item['updateTime'] = None
                try:
                    References = vuls[i].find(self.cvss + 'References').findall(self.cvss + 'Reference')
                    item['vendorPatch'] = ''
                    for reference in References:
                        if reference.find(self.cvss + 'URL').text !=None:
                            item['vendorPatch'] +=reference.find(self.cvss + 'URL').text+'\n'
                except:
                    item['vendorPatch'] = ''
                item['chinese'] = False
                item['total'] = self.totalVuls
                item['current'] = int(self.currentVuls)
                yield item

